{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple linear regression\n",
    "Solution using MLP and Linear Algebra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 8)                 16        \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 9         \n",
      "=================================================================\n",
      "Total params: 25\n",
      "Trainable params: 25\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Failed to import pydot. You must install pydot and graphviz for `pydotprint` to work.\n",
      "Train on 10 samples\n",
      "Epoch 1/100\n",
      "10/10 [==============================] - 1s 109ms/sample - loss: 8.4181\n",
      "Epoch 2/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 5.7370\n",
      "Epoch 3/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 3.8146\n",
      "Epoch 4/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 2.6083\n",
      "Epoch 5/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.8234\n",
      "Epoch 6/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.3077\n",
      "Epoch 7/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 0.9801\n",
      "Epoch 8/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.6866\n",
      "Epoch 9/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.4787\n",
      "Epoch 10/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 0.3478\n",
      "Epoch 11/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 0.2568\n",
      "Epoch 12/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.1964\n",
      "Epoch 13/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.1499\n",
      "Epoch 14/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 0.1252\n",
      "Epoch 15/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0894\n",
      "Epoch 16/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 0.0693\n",
      "Epoch 17/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0516\n",
      "Epoch 18/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0411\n",
      "Epoch 19/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0288\n",
      "Epoch 20/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0201\n",
      "Epoch 21/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0157\n",
      "Epoch 22/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0115\n",
      "Epoch 23/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 0.0090\n",
      "Epoch 24/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0068\n",
      "Epoch 25/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0047\n",
      "Epoch 26/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0036\n",
      "Epoch 27/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 0.0026\n",
      "Epoch 28/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0019\n",
      "Epoch 29/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 0.0014\n",
      "Epoch 30/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 0.0011\n",
      "Epoch 31/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 8.3436e-04\n",
      "Epoch 32/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 5.4558e-04\n",
      "Epoch 33/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 3.7619e-04\n",
      "Epoch 34/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 2.7842e-04\n",
      "Epoch 35/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 2.1498e-04\n",
      "Epoch 36/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.5641e-04\n",
      "Epoch 37/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 1.1316e-04\n",
      "Epoch 38/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 8.4790e-05\n",
      "Epoch 39/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 6.0916e-05\n",
      "Epoch 40/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 4.6339e-05\n",
      "Epoch 41/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 3.5867e-05\n",
      "Epoch 42/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 2.6099e-05\n",
      "Epoch 43/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.9969e-05\n",
      "Epoch 44/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 1.3725e-05\n",
      "Epoch 45/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 1.0290e-05\n",
      "Epoch 46/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 7.2686e-06\n",
      "Epoch 47/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 5.5383e-06\n",
      "Epoch 48/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 4.2897e-06\n",
      "Epoch 49/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 3.2522e-06\n",
      "Epoch 50/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 2.3353e-06\n",
      "Epoch 51/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 1.7212e-06\n",
      "Epoch 52/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.2761e-06\n",
      "Epoch 53/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 8.6673e-07\n",
      "Epoch 54/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 6.3416e-07\n",
      "Epoch 55/100\n",
      "10/10 [==============================] - 0s 992us/sample - loss: 4.9319e-07\n",
      "Epoch 56/100\n",
      "10/10 [==============================] - 0s 938us/sample - loss: 3.7122e-07\n",
      "Epoch 57/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 2.7542e-07\n",
      "Epoch 58/100\n",
      "10/10 [==============================] - 0s 942us/sample - loss: 2.0714e-07\n",
      "Epoch 59/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.4989e-07\n",
      "Epoch 60/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.1624e-07\n",
      "Epoch 61/100\n",
      "10/10 [==============================] - 0s 908us/sample - loss: 8.4808e-08\n",
      "Epoch 62/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 6.4244e-08\n",
      "Epoch 63/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 4.4389e-08\n",
      "Epoch 64/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 3.1544e-08\n",
      "Epoch 65/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 2.3182e-08\n",
      "Epoch 66/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.6740e-08\n",
      "Epoch 67/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.1735e-08\n",
      "Epoch 68/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 8.9919e-09\n",
      "Epoch 69/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 6.3282e-09\n",
      "Epoch 70/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 4.6842e-09\n",
      "Epoch 71/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 3.5122e-09\n",
      "Epoch 72/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 2.6531e-09\n",
      "Epoch 73/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 1.9445e-09\n",
      "Epoch 74/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.4403e-09\n",
      "Epoch 75/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.0559e-09\n",
      "Epoch 76/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 7.3598e-10\n",
      "Epoch 77/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 5.5197e-10\n",
      "Epoch 78/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 4.1046e-10\n",
      "Epoch 79/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 3.0287e-10\n",
      "Epoch 80/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 2.2959e-10\n",
      "Epoch 81/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 1.7719e-10\n",
      "Epoch 82/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.2162e-10\n",
      "Epoch 83/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 8.6062e-11\n",
      "Epoch 84/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 6.5083e-11\n",
      "Epoch 85/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 4.5488e-11\n",
      "Epoch 86/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 3.3987e-11\n",
      "Epoch 87/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 2.5058e-11\n",
      "Epoch 88/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.8902e-11\n",
      "Epoch 89/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 1.2513e-11\n",
      "Epoch 90/100\n",
      "10/10 [==============================] - 0s 2ms/sample - loss: 9.1873e-12\n",
      "Epoch 91/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 7.9140e-12\n",
      "Epoch 92/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 6.7146e-12\n",
      "Epoch 93/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 6.2997e-12\n",
      "Epoch 94/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 5.3575e-12\n",
      "Epoch 95/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 4.4082e-12\n",
      "Epoch 96/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 4.0487e-12\n",
      "Epoch 97/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 3.9037e-12\n",
      "Epoch 98/100\n",
      "10/10 [==============================] - 0s 3ms/sample - loss: 3.7019e-12\n",
      "Epoch 99/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 3.0823e-12\n",
      "Epoch 100/100\n",
      "10/10 [==============================] - 0s 1ms/sample - loss: 3.2315e-12\n",
      "k (Linear Algebra Method):\n",
      "[[2.]\n",
      " [3.]]\n",
      "Ground Truth, Linear Alg Prediction, MLP Prediction\n",
      "[[1.         1.         1.00000286]\n",
      " [1.4        1.4        1.400002  ]\n",
      " [1.8        1.8        1.80000138]\n",
      " [2.2        2.2        2.200001  ]\n",
      " [2.6        2.6        2.60000038]\n",
      " [3.         3.         2.99999976]\n",
      " [3.4        3.4        3.39999938]\n",
      " [3.8        3.8        3.79999876]\n",
      " [4.2        4.2        4.1999979 ]\n",
      " [4.6        4.6        4.59999752]]\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "# numpy package\n",
    "import numpy as np\n",
    "\n",
    "# keras modules\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.utils import plot_model\n",
    "\n",
    "# generate x data\n",
    "x = np.arange(-1,1,0.2)\n",
    "x = np.reshape(x, [-1,1])\n",
    "\n",
    "# generate y data\n",
    "y = 2 * x + 3\n",
    "\n",
    "# True if noise is added to y\n",
    "is_noisy = False\n",
    "\n",
    "# add noise if enabled\n",
    "if is_noisy:\n",
    "    noise = np.random.uniform(-0.1, 0.1, x.shape)\n",
    "    x = x + noise\n",
    "\n",
    "# deep learning method\n",
    "# build 2-layer MLP network \n",
    "model = Sequential()\n",
    "# 1st MLP has 8 units (perceptron), input is 1-dim\n",
    "model.add(Dense(units=8, input_dim=1))\n",
    "# 2nd MLP has 1 unit, output is 1-dim\n",
    "model.add(Dense(units=1))\n",
    "# print summary to double check the network\n",
    "model.summary()\n",
    "# create a nice image of the network model\n",
    "plot_model(model, to_file='linear-model.png', show_shapes=True)\n",
    "# indicate the loss function and use stochastic gradient descent\n",
    "# (sgd) as optimizer\n",
    "model.compile(loss='mse', optimizer='sgd')\n",
    "# feed the network with complete dataset (1 epoch) 100 times\n",
    "# batch size of sgd is 4\n",
    "model.fit(x, y, epochs=100, batch_size=4)\n",
    "# simple validation by predicting the output based on x\n",
    "ypred = model.predict(x)\n",
    "\n",
    "# linear algebra method\n",
    "ones = np.ones(x.shape)\n",
    "# A is the concat of x and 1s\n",
    "A = np.concatenate([x,ones], axis=1)\n",
    "# compute k using using pseudo-inverse\n",
    "k = np.matmul(np.linalg.pinv(A), y) \n",
    "print(\"k (Linear Algebra Method):\")\n",
    "print(k)\n",
    "# predict the output using linear algebra solution\n",
    "yla = np.matmul(A, k)\n",
    "\n",
    "# print ground truth, linear algebra, MLP solutions\n",
    "outputs = np.concatenate([y, yla, ypred], axis=1)\n",
    "print(\"Ground Truth, Linear Alg Prediction, MLP Prediction\")\n",
    "print(outputs)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
